const puppeteer = require('puppeteer');
const fs = require('fs');

let totalBlogs = [];
let errLogs = [];

let topics_map = {'11000115': '网络技术'};

async function main() {
  const fileName = `tuicool_topics.json`;
  let totalBlogs = [];

  const browser = await puppeteer.launch({headless: false, slowMo: 0});
  const page = (await browser.pages())[0];
  await page.setViewport({width: 1500, height: 0});

  page.on('pageerror', msg => {
    console.log(msg);
  });


  await page.goto('https://www.tuicool.com/login');
  await page.type('#xlEmail', 'yinhaixiang@qq.com');
  await page.type('#xlPassword', 'sean0410');
  await page.click('body > div.container-fluid > div.center_container.container-top > form > fieldset > div > div:nth-child(4) > button');
  await page.waitFor(1000);

  let blogs = null;

  for (let topics_id in topics_map) {
    for (let i = 1; i <= 20; i++) {
      let url = `https://www.tuicool.com/topics/${topics_id}?st=1&lang=1&pn=${i}`;
      await page.goto(url, {waitUntil: 'domcontentloaded'});
      await page.waitFor(1000);
      blogs = await page.$$eval('#list_article > div > div.aricle_item_info', async (eles, subTab) => {
        let tmpBlogs = [];
        eles.forEach(async ele => {
          let title = ele.querySelector('div.title > a').innerText;
          let content = ele.querySelector('div.title > a').href;
          let source = ele.querySelector('div.tip > span:nth-child(1)').innerText.trim();
          if (!source.includes("博客园")) {
            tmpBlogs.push({
              title: title,
              content: content,
              tab: 'tuicool',
              sub_tab: subTab,
              source: source
            });
          }
        });
        return tmpBlogs;
      }, topics_map[topics_id]);

      console.log(blogs);
      totalBlogs = totalBlogs.concat(blogs);
      console.log(topics_id + ',' + i + ',--------------');
    }
  }


  let writerStream = fs.createWriteStream(`/Users/yinhaixiang/WebstormProjects/egg-taosha/crawler/crawler_data/${fileName}`);
  writerStream.write(JSON.stringify(totalBlogs));
  console.log('write done');

  console.log(errLogs);

  // browser.close();
};


main();


